import logging
import re
import difflib
import sys
import string

from collections import defaultdict
from functions import *
from search import Search

vowels = r'[AEIOUY]'
consonants = r'[bdfghklmnpqrstwx]'
html = re.compile(r'<.*?>')
braces = re.compile(r'\[.*?\]')
parens= re.compile(r'\(.*?\)')
char_table = ''.join(chr(i) for i in xrange(256))

class Corpus:
    """A collection of documents"""
    def __init__(self):
        self.dict_raw = defaultdict(int)
        self.dict_snd = defaultdict(set)
        self.variants = defaultdict(set)
        self.models = []
        self.witnesses = []

    def add(self, word, sound):
        """Add a word and it's sound to the database"""
        self.dict_raw[word] += 1
        self.dict_snd[sound].add(word)

    def normalise(self, sounds1, sounds2):
        """To be removed"""
        target = sounds2[:]
        for s1 in sounds1:
            for i, s2 in enumerate(target):
                if s2 in self.variants[s1] and s2 not in sounds1:
                     target[i] = s1
        return target

    def list(self):
        """Returns all prayers in the corpus"""
        for witness in self.witnesses:
            for prayer in witness.list():
                yield prayer

    def search(self, list_or_string):
        """Search the corpus
           search criteria can be specified as [+-](type:)?word
           where type is the prayer model and word is the word to be searched"""
        if isinstance(list_or_string, list):
            string = ' '.join(list_or_string)
        else:
            string = list_or_string
        search = Search(string)
        logging.info(search)
        for witness in self.witnesses:
            if search.search(witness):
                yield witness

    def unsoundex(self, s):
        """Transform a soundex back to a word"""
        if self.dict_snd.has_key(s):
            return next(iter(self.dict_snd[s]))
        return None

    def classify(self, cutoff=9999):
        """Classify the instances in this corpus"""
        for p in self.list():
            p.classify(cutoff)

    def check(self, prnt = False):
        """Test method"""
        errors = prayers = 0
        for p in self.list():
           if p.type() != 'x' and p.assigned_type() != '?':
               if p.type() != p.assigned_type():
                   errors += 1; prayers += 1
               else:
                   prayers += 1
        print errors, prayers, float(errors)/prayers

def clean_encoding(encoding):
    """Clean the encodings from Sanne's database"""
    encoding = encoding.replace('<aantal onbekend>', '')
    encoding = encoding.split(';')[0]
    encoding = encoding.split(' ')[0]
    encoding = encoding.replace('x', '')
    return encoding

def clean_text(text):
    text = html.sub('', text)   
    text = braces.sub('', text)   
    text = parens.sub('', text)   
    return text

class Witness:
    """A Textual witness in a corpus, identified by signature and folia"""
    def __init__(self, corpus, signature, folia, text, encoding, 
                 rubric, copyist, remarks, literature):
        self.corpus = corpus
        self.signature = signature.strip()
        self.folia = folia.strip()
        self.text = text.strip()
        self.encoding = clean_encoding(encoding)
        self.rubric = clean_text(rubric.strip())
        self.copyist = copyist
        self.remarks = remarks
        self.literature = literature

        self.prayers = []
        self.props = {}
        self.index = len(self.corpus.witnesses)

        if self.rubric:
            r = Rubric(self, self.rubric)
            if r.sounds:
                self.prayers.append(r) 

        text = clean_text(text)
        text = braces.sub('', text)   
        text = parens.sub('', text)   
        #text = text.translate(None, string.punctuation).lower()
        for prayer in text.split('\x0b'):
            prayer = prayer.strip()
            if len(prayer):
                p = Prayer(self,prayer)
                if p.sounds:
                    self.prayers.append(p) 

    def key(self):
        return str(self.index)

    def check(self, prnt = False):
        """Test method"""
        d = len(self.prayers) - len(self.encoding)
        # I have found more prayers than are possible according to the database
        if d == 0:
            diff = levenshtein(self.encode(), self.encoding)
            return diff, len(self.encoding)
        else:
            return 0, 0

    def check2(self, prnt = False):
        """Test method"""
        d = len(self.prayers) - len(self.encoding)
        # I have found more prayers than are possible according to the database
        if d > 0:
           if prnt:
               print "Error in", self.signature, self.folia, \
                     ": length mismatch,", self.encoding, self.encode()
               for prayer in self.prayers:
                   prayer.check(prnt)
                   print '-------------------'
               print
           diff = levenshtein(self.encode(), self.encoding)
           return diff - d, len(self.encoding)
        # I have found less prayers than are possible according to the database
        if d < 0:
           if prnt:
               print "Error in", self.signature, self.folia, \
                     ": length mismatch,", self.encoding, self.encode()
               for prayer in self.prayers:
                   prayer.check(prnt)
                   print '-------------------'
               print
           diff = levenshtein(self.encode(), self.encoding)
           return diff - d , len(self.prayers)

        # Same number, now check if the encoding in the database
        # differs from the one I found myself
        encoding = self.encode()
        if self.encoding.upper() != encoding:
            if prnt:
                print "Error in", self.signature, self.folia, \
                  ": coding mismatch,", self.encoding, encoding
                for i, prayer in enumerate(self.prayers):
                    if not prayer.model or \
                        prayer.model.type() != self.encoding[i]:
                        prayer.check(prnt)
                        print '-------------------'
                print
            diff = levenshtein(self.encode(), self.encoding)
            return diff, max(len(self.encoding), len(self.prayers))
        # Exact match
        if prnt:
            print "Okay", self.signature, self.folia, self.encoding
            print
        return 0, len(self.encoding)
  
    def encode(self):
        """Return the encoding of the prayers
           Note, this is not the same as the database supplied field encoding"""
        return ''.join( [ p.type() for p in self.prayers ] )

    def list(self):
        """Returns all prayers"""
        for prayer in self.prayers:
            yield prayer

    def search(self, criteria):
        """Search this Witness
           search criteria can be specified as [+-](type:)?word
           where type is the prayer model and word is the word to be searched
           TODO: remove me"""
        for c in criteria:
            if not c.type: continue
            prayers = [ p for p in self.prayers if p.type() == c.type ]
            if not prayers: return False
            for p in prayers:
                if not c.search(p.sounds):
                   return False

        sounds = []
        for p in self.prayers:
            sounds.extend(p.sounds)

        for c in criteria:
            if not c.type and not c.search(sounds): return False
        return True

class Prayer:
    """A prayer in the corpus. TODO: rename to instance"""
    def __init__(self, witness, text):
        self.witness = witness
        self.text = text
        self.corpus = witness.corpus
        self.words = []
        self.sounds = [] 
        self._set()
        self.model = None
        self.ratio = 0.0
        self.nearest_model = None
        self.nearest_letter = 'x'
        self.nearest_ratio = 0.0
        self.index = len(witness.prayers)

    def key(self):
        return self.witness.key() + '.' + str(self.index)

    def __str__(self):
        return "[%s %s %i %s] %s" % (self.witness.signature, self.witness.folia, self.witness.prayers.index(self), self.type(), self.text)

    def __repr__(self):
        return self.__str__()

    def _set(self):
        text = self.text.encode('utf-8').translate(char_table, string.punctuation).lower()
        for raw in text.split():
            sound = soundex(raw)
            # Interpunction or junk
            if sound[0] == '0': 
                continue

            self.sounds.append(sound)
            self.words.append(raw)
            self.corpus.add(raw, sound)

    def search(self, criteria):
        """TODO: remove this method. Should be in the SearchCriterion class"""
        for c in criteria:
            if c.type and c.type != self.type(): continue
            if c.opcode == '+' and c.phrase not in self.sounds: return False
            if c.opcode == '-' and c.phrase in self.sounds: return False
        return True

    def type(self):
        """Return the type of this Instance"""
        return self.model.type() if self.model else 'x'

    def assigned_type(self):
        """Return the type that was assigned by the students"""
        if len(self.witness.encoding) == len(self.witness.encode()):
            return self.witness.encoding[ self.witness.prayers.index(self) ]
        else:
            return '?'

    def features(self):
        """Return the featureset for this Instance"""
        return set(self.sounds)

    def unsoundex(self, sound):
        index = self.sounds.index(sound)
        return self.words[index]

    def check(self, prnt = False):
        if self.model is None:
            print self.witness.signature, self.witness.folia 
            print "No match found.",
            if self.nearest_model:
                print "Maybe Type", self.nearest_letter, self.nearest_ratio
                print `self.nearest_model.text`
            print `self.text`
            return False
        else:
            print self.witness.signature, self.witness.folia, \
              "\tType:", \
              self.type(), self.ratio
            print `self.model.text`
            print `self.text`
            return True

    def compare(self, model):
        f1 = model.unique()
        f2 = self.features()
        intersect = f1 & f2
        score = len(f2) - len(intersect)
        return score

    def lev_ratio(self, model):
        if not len(self.sounds): return 1.0
        dist = levenshtein( model.sounds, self.sounds )
        return dist / float(len(self.sounds))

    def jacard(self, other):
        intersect = self.features & other
        union = self.features | other
        return float(intersect) / float(union)
    
    def classify(self, cutoff=9999):
        best_score = 9999
        best_guess = None
        for j, model in enumerate(self.corpus.models):
            score = self.compare(model)
            if score < best_score:
                best_guess = j
                best_score = score

        if best_guess is None:
            return

        ratio = self.lev_ratio( self.corpus.models[best_guess] )
        if ratio < cutoff:
            self.model = self.corpus.models[best_guess]
            self.ratio = ratio
            self.nearest_letter = chr(ord('A') + best_guess)
            self.nearest_model = self.corpus.models[best_guess]
            self.ratio = ratio
        else:
            self.model = None
            self.ratio = ratio
            self.nearest_letter = chr(ord('A') + best_guess)
            self.nearest_model = self.corpus.models[best_guess]
            self.nearest_ratio = ratio

    def simplify(self, sounds = None, words = None):
        if not sounds:
            sounds = self.sounds[:]
            words = self.words[:]
        if not self.model: 
            return sounds, words
        other = self.model
        s = diff( other.sounds, sounds )
        j3 = 0 # If we do anything to update the length of the target array
        for tag, i1, i2, j1, j2 in s.get_opcodes():
            if tag == 'replace' and i2 - i1 == 2 and j2 - j1 == 1:
                # check for enclyse
                s1 = ''.join(other.words[i1:i2])
                s2 = self.words[j1]
                if self.approximate(s1, s2):
                   sounds[j1+j3:j2+j3] = other.sounds[i1:i2]
                   words[j1+j3:j2+j3] = other.words[i1:i2]
                   j3 += 1
            elif tag == 'replace' and i2 - i1 == 1 and j2 - j1 == 2:
                # check for enclyse
                s1 = other.words[i1]
                s2 = ''.join(self.words[j1:j2])
                if self.approximate(s1, s2):
                   sounds[j1+j3:j2+j3] = other.sounds[i1:i2]
                   words[j1+j3:j2+j3] = other.words[i1:i2]
                   j3 -= 1
            if tag == 'replace' and i2 - i1 < 10 and j2 - j1 < 10:
               for i4, w1 in enumerate(other.words[i1:i2]):
                   for j4, w2 in enumerate(self.words[j1:j2]):
                       if self.approximate(w1,w2):
                           sounds[j1+j3+j4] = other.sounds[i1+i4]
                           words[j1+j3+j4] = other.words[i1+i4]
                       if other.isVariant(i1+i4, sounds[j1+j3+j4]):
                           sounds[j1+j3+j4] = other.sounds[i1+i4]
                           words[j1+j3+j4] = other.words[i1+i4]
        return sounds, words

    def simplify2(self, sounds = None, words = None): 
        if not sounds:
            sounds = self.sounds[:]
            words = self.words[:]
        if not self.model: 
            return sounds, words
        other = self.model
        s = diff( other.sounds, sounds )
        for tag, i1, i2, j1, j2 in s.get_opcodes():
            if tag == 'replace' and i2 - i1 == 1 and j2 - j1 == 1:
                if not self.approximate(words[j1], other.words[i1]):
                    self.model.addVariant(self, i1, sounds[j1], words[j1])
        return sounds, words

    def isSimple(self, sounds, words):
        if not sounds:
            sounds = self.sounds[:]
            words = self.words[:]
        if not self.model: 
            return True
        other = self.model
        s = diff( other.sounds, sounds )
        for tag, i1, i2, j1, j2 in s.get_opcodes():
            if tag == 'replace' and i2 - i1 != 1 or j2 - j1 != 1:
                return false

    def approximate(self, w1, w2):
        s1 = soundex(w1)
        s2 = soundex(w2)
        if s1 == s2: return True
        lev = levenshtein(s1, s2)
        if lev < 2:
            if len(s1.strip('0')) > 3 and len(s2.strip('0')) > 3:
                return True
            # Check if different starting vowels
            if s1[0] in vowels and s2[0] in vowels \
               and s1[0] != s2[0]: return True
            # Check addition of final r or n before e
            if w1.endswith('er') and w2.endswith('e'): return True
            if w1.endswith('e') and w2.endswith('er'): return True
            if w1.endswith('en') and w2.endswith('e'): return True
            if w1.endswith('en') and w2.endswith('en'): return True
            if w1.endswith('e') and w2.endswith('en'): return True
            if s1[0] == 'H':
                 if s1 == soundex('h'+w2): return True
            if s2[0] == 'H':
                 if s2 == soundex('h'+w1): return True
        elif lev == 3:
            if s1[0] in vowels and s2[0] in vowels \
                and s1[0] != s2[0]: 
                if w1.endswith('er') and w2.endswith('e'): return True
                if w1.endswith('e') and w2.endswith('er'): return True
                if w1.endswith('en') and w2.endswith('e'): return True
        else:
            return False

class Rubric(Prayer):
    def __init__(self, witness, text):
        Prayer.__init__(self, witness, text)

    def type(self):
        return 'R'

    def assigned_type(self):
        return 'R'

class SearchCriterion:
    def __init__(self, string):
        self.string = string
        self.type = None
        if string.startswith('-'):
            self.opcode = '-'
            phrase = string[1:]
        elif string.startswith('+'):
            self.opcode = '+'
            phrase = string[1:]
        else:
            self.opcode = '+'
            phrase = string
        
        if phrase.find(':') != -1:
            type, phrase = phrase.split(':')
            self.type = type.upper()
        if len(phrase) == 7 and phrase[6].isdigit():
            self.phrase = phrase
        else:
            self.phrase = soundex(phrase)

    def __repr__(self):
        type = self.type if self.type else '*'
        return self.opcode + type + ':' + self.phrase 

    def search(self, witness):
        if self.type:
            prayers = [ p for p in witness.prayers if p.type() == self.type ]
        else:
            prayers = witness.prayers
        if not prayers: return False
        for p in prayers:
            if self.phrase in p.sounds and self.opcode == '+': return True
            if self.phrase in p.sounds and self.opcode == '-': return False
        if self.opcode == '-': 
            return True
        else:
            return False

class Model(Prayer):
    def __init__ (self, corpus, text):
        self.witness = None
        self.corpus = corpus
        self.text = text
        self.words = []
        self.sounds = [] 
        self._set()
        self.synonyms_snd = defaultdict(dict) # variants[index][prayer] = sound
        self.synonyms_wrd = defaultdict(dict) # variants[index][prayer] = word
        self._variants = None

    def unique(self):
        """Return the set of unique features of this model"""
        s = set(self.features())
        for o in self.corpus.models:
            if not o is self:
                s = s - o.features()
        return s

    def type(self):
        return chr(ord('A') + self.corpus.models.index(self))

    def common(self):
        """Return the set of all features of the instances of this model"""
        s = set(self.features())
        for p in self.corpus.list():
            if p.model is self:
               s = s + p.features()
        return s    

    def isVariant(self, index, sound):
        """Check if SOUND is a variant for the word at INDEX"""
        # TODO: maybe add a check. A SOUND can only be a synonym if 
        # it does not occur at another place in the model.
        # Also, probably count does not work, since count uses is and not ==
        d = self.synonyms_snd[index]
        if d.values().count(sound) > 3:
            return True
        else:
            return False

    def addVariant(self, prayer, index, sound, word):
        """Add a variant for this word"""
        self.synonyms_snd[index][prayer] = sound
        self.synonyms_wrd[index][prayer] = word

    def _variants2(self):
        diffs = defaultdict(list)
        counts = defaultdict(int) # counts of the word at position i

        def pretty(d):
            return '{'+','.join(['%s:%i' % (s,i) for s,i in d.iteritems()])+'}'
            
        def reverse(d):
            d2 = defaultdict(list)
            for p in d: 
                k = d[p]
                d2[k].append(p)
            return d2

        # First simplify
        for i in range(0,5):
            l = list(self.corpus.list())
            for j, p in enumerate(l):
                if p.model is self:
                    sounds, words = p.simplify()
                    new_sounds, new_words = p.simplify2(sounds, words)

        for p in self.corpus.list():
             if p.model is self:
                  # This removes enclyses and approximations 
                  sounds, words = p.simplify()
                  sounds, words = p.simplify(sounds, words)
                  s = diff( self.sounds, sounds )
                  l = s.get_opcodes()
                  for tag, i1, i2, j1, j2 in l:
                      diffs[i1].append((tag, i1, i2, j1, j2, p, words, sounds))
                      if tag == 'equal':
                          for i, word in enumerate(words[i1:i2]):
                              counts[i1+i] += 1

        for i, word in enumerate(self.words):
            synonyms = reverse(self.synonyms_wrd[i])
            inserts = defaultdict(list)
            deletes = defaultdict(list)
            replace = defaultdict(list)

            for d in diffs[i]:
                tag, i1, i2, j1, j2, p, w, s = d
                if tag == 'insert':
                    inserts[' '.join(w[j1:j2])].append((p, i1, i2, j1, j2))
                if tag == 'replace' and (i2 - i1 != 1 or j2 - j1 != 1):
                    replace[' '.join(self.words[i1:i2]) + '/' + ' '.join(w[j1:j2])].append((p, i1, i2, j1, j2))
                if tag == 'delete':
                    deletes[' '.join(self.words[i1:i2])].append((p, i1, i2, j1, j2))
            #for index, word, synonyms, count, inserts, deletes, replace in variants:
            yield i + 1, word, synonyms, counts[i], inserts, deletes, replace
        for d in diffs[i+1]:
            tag, i1, i2, j1, j2, p, w, s = d
            if tag == 'insert':
                inserts[' '.join(w[j1:j2])].append((p, i1, i2, j1, j2))
        yield i + 2, None, None, None, inserts, None, None
        
    def variants(self):
        """Return a list of all variants of this model"""
        if not self._variants:
            self._variants = list(self._variants2())
        return self._variants

    def diffs(self, other):
        return [s.get_opcodes()]

class Classification:
    """TODO: use me"""
    def __init__(self):
        self.items = set()

    def add(self, type, score):
        self.items.append((score, type))

    def sort(self):
        self.items.sort()

def diff(a,b):
    s = difflib.SequenceMatcher(None, a,b)
    return s
